import re #regular expression library
import sys #writing to files
import urllib #url library
import string #string functions
import cgi

def getUrlData(url):
    #TO-DO check connection
    urlSearch = url
    urlfile = urllib.urlopen(urlSearch)
    
    #urldata will contain the url data with out null chars
    urldata = urlfile.read().replace("\0","")
    urlfile.close()
    return urldata

def extractResultsLive(urldata):
    
    regexGetResultBlock = "\<div id=\"results\"\>.+?\</div\>"
    resultBlock = re.findall(regexGetResultBlock, urldata, re.DOTALL)
    resultBlockAsString = ''.join(resultBlock) #maxim wants to ask carpenter
    
    regexGetResultsUrls = "(?<=\<h3\>\<a href=\").+?(?=\")"
    resultsUrls = re.findall(regexGetResultsUrls, resultBlockAsString, re.DOTALL)
     
    return resultsUrls
#end extractResultsLive    
        
def extractResultsYahoo(urldata):  
    
    
    regexGetResultBlock = "\<h2\>WEB RESULTS\</h2\>.+?\</ol\>"
    resultBlock = re.findall(regexGetResultBlock, urldata, re.DOTALL)
    resultBlockAsString = ''.join(resultBlock) #maxim wants to ask carpenter
          
    regexGetResultsUrls = "(?<=\<a class=yschttl href=\").+?(?=\")"
    resultsUrls = re.findall(regexGetResultsUrls, resultBlockAsString, re.DOTALL)
    
    return redirectedUrls(resultsUrls)
    
#end extractResultsYahoo

def getNextUrlYahoo(urldata):
    regexGetNextUrl = "(?<=id\=\"pg-next\" class\=\"pg\" href\=\").+?(?=\"\>Next)"
    nextUrl = re.findall(regexGetNextUrl, urldata, re.DOTALL)
    if len(nextUrl) != 0:
        return nextUrl[0]
    else:
        return ""
#end getNextUrlYahoo

def getNextUrlLive(urldata):   
    regexGetNextUrl = "(?<=class\=\"nP\" href\=\").+?(?=\"\>Next)"    
    nextUrl = re.findall(regexGetNextUrl, urldata, re.DOTALL)  
    if len(nextUrl) != 0: 
        fixNextUrl = string.replace(nextUrl[0], ";", "")
        fixNextUrl = string.replace(fixNextUrl, "amp", "")
        fixNextUrl = "http://search.live.com" + fixNextUrl 
        return fixNextUrl
    else:
        return ""
#end getNextUrlYahoo


def redirectedUrls(urls):
    count = 0
    for url in urls:
        if string.find(url, "http://rdre") != -1:
            urlfile = urllib.urlopen(url)
            urls[count] = urlfile.geturl()
        count = count + 1
    return urls
#end redirectedUrls
  
def insertResultsToDictionary(dictionary, results):
    for result in results:
        if dictionary.has_key(result):
            dictionary[result] = 2
        else:
            dictionary[result] = 1
#end insertResultsToDictionary

def findMatches(dictionary):
    matches = []
    for url, value in dictionary.iteritems():
        if value > 1:
            matches.append(url)
    
    return matches
#end findMatches

def printResponse(matchesFound):
	thepage = '''<html><head>
	<title>%s</title>
	</head>
	<body bgcolor="MistyRose">
	<h3>%s</h3>
	<p>
	%s
	</p>
	<a href="http://localhost/Python/"><b>Try a different query</b></a>
	</body></html>
	'''

	title = 'OPIM 311 Final Project - Maxim Novak & Inbal Doron'
	
	heading = 'Your Matches:'
	
	text = ''
	for match in matchesFound:
		text += "<p><a href="
		text += match
		text += ">"
		text += match
		text += "</a></p>"
		
	print "Content-Type: text/html"
	print
	print thepage % (title, heading, text)
#end printResponse

#MAIN

targetMatches = 3

form = cgi.FieldStorage() 
if form.has_key('query'):
    textQuery = form["query"].value
else:
	textQuery = ""

query = urllib.quote(textQuery) #replaces unsafe and special chars with %ascii 
#query = "nike"

resultsDictionary = {}

matchesFound = []

liveData = getUrlData("http://search.live.com/results.aspx?q=" + query)
firstResultsLive = extractResultsLive(liveData)  

yahooData = getUrlData("http://search.yahoo.com/search?p=" + query)
firstResultsYahoo = extractResultsYahoo(yahooData)

insertResultsToDictionary(resultsDictionary, firstResultsLive)

insertResultsToDictionary(resultsDictionary, firstResultsYahoo)

matchesFound = findMatches(resultsDictionary)

yahooNext = getNextUrlYahoo(yahooData)
liveNext = getNextUrlLive(liveData)

while (len(matchesFound) < targetMatches) and (yahooNext != "" or liveNext != ""):
    if yahooNext != "":
        yahooData = getUrlData(yahooNext)
        resultsYahoo = extractResultsYahoo(yahooData)
        insertResultsToDictionary(resultsDictionary, resultsYahoo)
        yahooNext = getNextUrlYahoo(yahooData)
    if liveNext != "":
        liveData = getUrlData(liveNext)
        resultsLive = extractResultsLive(liveData)
        insertResultsToDictionary(resultsDictionary, resultsLive)
        liveNext = getNextUrlLive(liveData)
    matchesFound = findMatches(resultsDictionary)
    
printResponse(matchesFound)

#end MAIN
    